from BeautifulSoup import *
from urlparse import *
from pysqlite2 import dbapi2 as sqlite

#       creating database

dbname = 'database_test_4'
def separatewords(text):
	 splitter = re.compile('\\W*')
	 return [s.lower( ) for s in splitter.split(text) if s!='']

def remPort(url):
	l = url.split(':')
	if len(l) > 2 :
		t = l[2].find('/')
		url = l[0] + ':' + l[1] + l[2][t:]
	return url

def create_db(name):
	db = sqlite.connect(name)
	db.execute('create table urllist(url,docid, num_outlinks integer)')
        db.execute('create table link(fromid, toid)')
        db.execute('create index urlidx on urllist(url)')
        db.execute('create index docidx on urllist(docid)')
        db.execute('create index urltoidx on link(toid)')
        db.execute('create index urlfromidx on link(fromid)')
        db.commit()

#create_db(dbname)
#db = sqlite.connect(dbname)

fp = open('B07','r')
#fp = fp.encode('iso-8859-1','replace')
data = fp.read()
#data = unicode(data,errors='ignore')
#print data
try :
	soup = BeautifulSoup(data)
except :
	data = unicode(data,errors='ignore')
	soup = BeautifulSoup(data)
#print soup.prettify()
docs = soup.contents
#print soup.string
#print len(docs)
#print docs[3]
for i in range(0,len(docs),2):
	docid = docs[i].docno
	x = str(docs[i].dochdr)
	t = x[8:].split(' ')
	docurl = t[0]
	docurl = remPort(t[0]).strip();
	print docurl

#	text = docs[i].findAll(text=True)
#	for k in text:
#		x = separatewords(k)
#		for tt in x:
#			print tt
	list = docs[i]('a')
#	print list
	for j in list:
		v = ''
		if ('href' in dict(j.attrs)):
			m = j['href']
			n = urljoin(docurl,m)
			try :
				print n
			except :
			 	continue
#			n = n.replace('"','')
#			n = n.replace("'","")
#			if m[0:4] == 'http':
#			try:
	#			m = remPort(m)
#			print '+++++++++++++++++++++\t' + n + '\t' + str(len(n))
#			except:
#				m = repr(m)
#				temp = m.split('>')
#				m = temp[0]
#				m = m.replace("u'","")
#				print m
#			query1="insert into link (fromid,toid) values(\""+str(docurl)+"\",\""+str(n)+"\")"
#			db.execute(query1)
	print "\n#########################################################################"
#db.commit()
